{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def nn(x, w): return np.sign(np.dot(x, w)).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = np.array([[-2,4,-1], [4,1,-1], [1, 6, -1], [2, 4, -1], [6, 2, -1]])\n",
    "t = np.array([-1, -1, 1, 1, 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7fed4d048198>]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD8CAYAAABXe05zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAIABJREFUeJzt3Xd4VHX+xfH3Z1IIhN47iDSRbqiBxEIHQREbCooFFZDmrq6r7k/dYltDEQURxYYNFESQEiwJoZrQpCpFFKREUYpI//7+SOKyLsgAmdyZyXk9Tx5JvGTOQznc3LlzYs45REQkdPi8DiAiImdHxS0iEmJU3CIiIUbFLSISYlTcIiIhRsUtIhJiVNwiIiFGxS0iEmJU3CIiISYyEJ+0dOnSrnr16oH41CIiYSkjI+MH51wZf44NSHFXr16d9PT0QHxqEZGwZGZb/T1Wl0pEREKMiltEJMSouEVEQoyKW0QkxKi4RURCjIpbRCTEqLglV/Wb3Y9+s/t5HUMkrKm4RURCjF/FbWbFzWyKma03s3Vm1irQwURE5NT8feXkKGC2c66XmUUDhQKYSURE/sAZi9vMigIJwK0AzrkjwJFAhNm59xBJyRt4oFNdShUuEIiHkFz2++vZ6bvST/nxiZ0m5lkmkXDnz6WSGkAmMNHMlpvZBDOL/f1BZtbfzNLNLD0zM/OcwqRv3cPU5dtpl5TChyu245w7p88jIhLO7EzlaGZxwGIg3jm3xMxGAfucc4+c7ufExcW5cx2Z2rBzP/e/v4qV3/3MFXXL8o+r61OhWMFz+lyS93LOtHWGLXJ2zCzDORfnz7H+nHFvA7Y555Zkvz8FaHqu4c6kTvkifHBPax7uehELNv1A+6RUJi3ZyokTOvsWEQE/its5txP4zszqZH/oCmBtIENF+Iw72tZgztAEGlQqxkNTV9N7wmK++eGXQD6siEhI8Pc+7nuBSWa2CmgM/Ctwkf6jWqlY3rqzBU/2bMCa7fvoODKV8ambOHb8RF48vIhIUDrjNe5zcT7XuE9n595DPDztS+at202jysV4qldD6pYvmquPISLildy+xh0UyheL4aW+cTx3YxO2/fQr3UankZT8FYePHfc6mohIngqZ4gYwM65sVJHk4Yl0a1iB0Z98zZXPpbH825+8jiYikmdCqrhzlIyNZuQNTXjl1jj2HzpGz7EL+fuMtRw8cszraCIiAReSxZ3j8rrlmDssgd7Nq/Jy2hY6jZzPwo0/eB1LRCSgQrq4AYrERPHPqxvwTv+W+Ax6T1jCX95fxd5fj3odTUQkIEK+uHO0rFGKWUMSuCuhBu+lf0eHESkkr93ldSwRkVwXNsUNUDA6gge7XMS0gfGUKBTNna+nM+itZfxw4LDX0UREck1YFXeOhpWLM31QG4a3r82cNTtpn5TCtOUarRKR8BCWxQ0QHelj8BW1mDm4LdVKxTL03RXc/lo63//8q9fRRETOS9gWd47a5Yrw/j2teaRbPRZt+pEOI1J5c7FGq0QkdIV9cUPWaNXtbS5gztAEGlUpxsPTVnPjS4vZotEqEQlB+aK4c1QtVYg3b2/BU9c0YO2OfXQamcqLKRqtEpHQkq+KG7JeNn99s6rMG55IQu0yPDFrPT3HLmTdjn1eRxMR8Uu+K+4c5YrGML7PJYzp3YTtP/3Klc+lkTR3g0arRCTo5dvihqyz724NKzJveCJXNqrI6E830m10Gss0WiUiQSxfF3eOErHRjLi+MRNvbcaBw8e4ZuxCHv9Io1UiEpxU3Ce5rG5Z5g5L4KYWVXllwRY6jkxlgUarRCTIqLh/p0hMFP+4qgHv9m9JpM/HTROW8MAUjVaJSPBQcZ9GixqlmDWkLXcl1mByxne0T0ph7pqdXscSEVFx/5GYqAge7Jw1WlUyNpr+b2Qw8K1lZO7XaJWIeEfF7YeGlYvz0b1tuK99bZLX7KL9iBSmLt+m0SoR8YSK209RET7uvaIWMwe34YLSsQx7dyX9Xv2C7RqtEpE8puI+S7XKFWHK3a35W7d6LNm8hw5JKbyh0SoRyUMq7nMQ4TNua3MBc4cl0KRqCR6Ztpobxi9mc+YBr6OJSD6g4j4PVUoW4o3bm/P0NQ1Zt3MfnUfNZ5xGq0QkwFTc58nMuK5ZFeYNTySxdhmenLWeq15YwNrvNVolIoHhV3Gb2Tdm9qWZrTCz9ECHCkXlisbwYp9LeL53U3buPUT3MWk8q9EqEQmAsznjvsw519g5FxewNCHOzOjasALJwxLp3rgiz326ka6j08jYqtEqEck9ulQSACVio0m6rjET+zXj4OFj9Bq3kMc+WsMvhzVaJSLnz9/idsBcM8sws/6BDBROLqtTlrnDE+nTshoTF3xDx5GpzP860+tYIhLi/C3ueOdcU6AzMNDMEn5/gJn1N7N0M0vPzFQ55ShcIJLHe9TnvbtaERXho8/LS7l/ykr2HtRolYicGzvbl22b2aPAAefcv093TFxcnEtP13OYv3fo6HFGffI141M3UzI2mr/3qE+n+uW9jiUiQcDMMvx9DvGMZ9xmFmtmRXJ+DHQAVp9fxPwpJiqCBzrVZdqAeEoXLsDdb2YwcJJGq0Tk7PhzqaQckGZmK4GlwEzn3OzAxgpvDSoXY/qgeP7csQ7Ja7NGqz5YptEqEfHPWV8q8Yculfhv4+793D9lFcu+/ZlL65Thn1c3oFLxgl7HEpE8lquXSiSwapYtwuS7W/PolfVYuiV7tGrRNxqtEpHTUnEHgQifcWv8BcwZmkDTaiV45MM1Gq0SkdNScQeRKiUL8fptzXmmV0PW79xHp1HzGfu5RqtE5L+puIOMmXFtXNZo1WV1yvDUbI1Wich/U3EHqbJFY3ixTxxjb2rKzr2H6T4mjX/P2cChoxqtEsnvVNxBrnODCswbnkCPxpUY89lGuo6eT8bWPV7HEhEPqbhDQPFC0Tx7XSNeu605h46eoNe4RTw6XaNVIvmVijuEJNYuw5xhCfRtWY1XF2q0SiS/UnGHmMIFInmsR30m392K6Mis0ao/T9ZolUh+ouIOUc2ql+TjwW0ZcOmFfLB8O+1GpDB79U6vY4lIHlBxh7CYqAju71SXDwfGUyZ7tGrApAx27z/kdTQRCSAVdxioX6kYH2aPVs1bt5v2Sam8n6HRKpFwpeIOE1ERPgZeVpOPB7elZtnC3Dd5JbdM/IJtPx30OpqI5DIVd5ipWbYwk+9qxWPdLyb9mz10HJHK64s0WiUSTlTcYcjnM25pXf230aq/fbiG68cvYpNGq0TCgoo7jOWMVv372kZ8tesAnUfN54XPN3JUo1UiIU3FHebMjF6XVCZ5eAJX1C3L07M3cNXzC1i9fa/X0UTkHKm484myRWIYe/MljL2pKbv2HabH8wt4Zs56jVaJhCAVdz6TM1p1dZNKPP/ZJrqMnk/6NxqtEgklKu58qHihaP59bSNev605h4+e4NoXNVolEkpU3PlYQu0yzB2WwC2tqvPaom/oMCKV1K80WiUS7FTc+VxsgUge7X4xk+9qRYEoH31fWcqfJq/k54NHvI4mIqeh4hYA4rJHqwZediFTl2+nXVIqs77c4XUsETkFFbf8JiYqgj93rMv0QfGUK1qAeyYt4543NVolEmxU3PI/Lq5YjGkD47m/Ux0+WZ81WjU5/TuNVokECRW3nFJUhI8Bl9Zk1pC21C5XmD9PWUXfV5by3Z7QG63a2qcvW/v09TqGSK7xu7jNLMLMlpvZjEAGkuByYZnCvNu/FY/3uJhlW3+i48hUXl2wRaNVIh46mzPuIcC6QAWR4OXzGX1bVWfOsATiqpfk0Y/Wct2Li9i4W6NVIl7wq7jNrDLQFZgQ2DgSzCqXKMRr/Zrx7LWN+Hr3AbqMms/zn2m0SiSvRfp53EjgfqBIbj749S8u8uu4d+9qlZsPK+fBzLjmksok1C7D/01fzTNzNjBz1Q6e7tWQ+pWKeR0P4H+uZx/84otTfrzaG6/nWSaR3HTGM24z6wbsds5lnOG4/maWbmbpmZl69V24K1OkAC/cdAnjbm5K5oGs0aqnZmu0SiQv2Jlu8TKzJ4A+wDEgBigKfOCcu/l0PycuLs6lp6fnZk4JYnsPHuUfM9cyOWMbNUrH8lSvhjSrXtLrWL/JOdPWGbYEMzPLcM7F+XPsGc+4nXMPOucqO+eqAzcAn/5RaUv+U6xQFM9c24g3bm/OkeMnuHbcIv724WoOaLRKJCB0H7fkmra1yjBnaAK3tq7OG4u30nFEKikarRLJdWdV3M65z51z3QIVRkJfzmjVlLtbERPl45ZXljL8vRUarRLJRWe8xn0udI1bAA4dPc6YTzcyLmUTxQtF8XiP+nRpUMHrWCJBKVevcYucq5ioCP7UsQ4fDoqnfLEYBkxaxt1vZLB7n0arRM6HilsC7uKKxZg2IJ4HOtXl0w27aZeUwnsarRI5ZypuyRORET7uufRCZg9pS93yRbk/hEerRLym4pY8VaNMYd7p35K/nzRaNXHBFo5rtErEbypuyXM+n9GnVXXmDk+kWfWSPPbbaNV+r6OJhAQVt3imUvGCvNqvGUnXNWJT5gG6jEpjzKdfa7RK5AxU3OIpM6Nn08okD0uk/cXl+Pfcr+g+ZgGrt+/1OppI0FJxS1AoU6QAz/duyot9LuGH7NGqJ2dptErkVFTcElQ6XlyeecMS6dW0MuNSNtFl1HyWbtnjdSyRoKLilqBTrFAUT/VqyJu3t+DI8RNc9+IiHpmm0SqRHCpuCVptapVm7rAEbou/gDeXbKVDUgqfbdjtdSwRz6m4JagVio7kb1fWY8rdrSlUIJJ+E79g+Lsr+OkXjVZJ/qXilpBwSbUSzBzchsGX12T6yu9pPyKFmat26GXzki+puCVkFIiMYHiHOkwf1IYKxQoy8K1l3KXRKsmHVNwScupVLMrUAa15sHNdUr7K5IqkFN77QqNVkn+ouCUkRUb4uCvxQmYNactFFYpy//ur6POyRqskf1BxS0irUaYw79zZkn9cVZ8V3/1MhxGpvJKm0SoJbypuCXk+n3Fzy2rMHZZAixoleXzGWq4dt5Cvd2m0SsKTilvCRsXiBZl4azNGXt+YLT/8QtfRaTz3iUarJPyouCWsmBlXNalE8vBEOlxcjmeTv+LK59L4cptGqyR8qLglLJUuXIAxvZsyvs8l7PnlCD2eT+OJWes0WiVhQcUtYa3DxeVJHp7IdXFVeDFlM51HzWfJ5h+9jiVyXlTcEvaKFYziyWsaMumOFhw7cYLrxy/m4Wlfsv/QUa+jiZwTFbfkG/E1SzNnaAK3t7mASUu+peOIVD5br9EqCT0qbslXCkVH8ki3erx/T2tiC0TS79UvGPbuCvZotEpCiIpb8qWmVUswY3AbBl9Ri49Wfk/7pBRmrPpeL5uXkHDG4jazGDNbamYrzWyNmT2WF8FEAq1AZATD29fmo3vbUKlEQQa9tZz+b2SwS6NVEuT8OeM+DFzunGsENAY6mVnLwMYSyTsXVSjKB/e05q9d6pL6VSbtklJ494tvdfYtQeuMxe2yHMh+Nyr7TX+iJaxERvjon3Ahc4YmUK9CUR54/0tumrCEb3/UaJUEH7+ucZtZhJmtAHYDyc65Jac4pr+ZpZtZemZmZm7nFMkT1UvH8vadLfnn1fVZtW0vHUem8rJGqyTI2Nl8OWhmxYGpwL3OudWnOy4uLs6lp6fnQjwR7+zY+ysPTV3Np+t307hKcZ7u1ZDa5Yp4HUvClJllOOfi/Dn2rO4qcc79DHwOdDqHXCIhpUKxgrx8SxyjbmjM1h9/oevo+Yz+5GuOHNNolXjLn7tKymSfaWNmBYF2wPpABxMJBmZGj8aVmDc8kU71K5CU/BXdx6Sx8rufvY4m+Zg/Z9wVgM/MbBXwBVnXuGcENpZIcClVuADP3diEl/rG8dPBI1z9wgKe+Hgdvx7RaJXkvcgzHeCcWwU0yYMsIkGvfb1ytKhRkic+XseLqZuZs2YnT17TkJY1SnkdTfIRvXJS5CwVjYniiZ4NeeuOFpxwcMP4xTw0VaNVkndU3CLnqHX2aNUdbS7g7aXf0mFEKp+u3+V1LMkHVNwi56FgdAQPZ49WFYmJ5LZX0xn6znKNVklAqbhFckGTqiWYcW9bhlxRi5lf7qBdUgrTV2q0SgJDxS2SS6IjfQzLHq2qUqIgg99ezp2vZ7Bzr0arJHepuEVyWd3yRflgQDwPdbmItI2ZtE9K4e2lGq2S3KPiFgmACJ9xZ0INZg9J4OJKRXnwgy/p/dIStv74i9fRJAyouEUCqHrpWN66oyX/uroBq7dnjVZNmL9Zo1VyXlTcIgHm8xm9W1Rl7vAE4i8szT9mruOasQv5atd+r6NJiFJxi+SRCsUKMiF7tOrbPQfpOno+o+ZptErOnopbJA/ljFYlD0ugS4MKjJin0So5eypuEQ+UKlyAUTc0YULfOH4+eJSrX1jAvzRaJX5ScYt4qF29cswdnsANzasyPnUznUelsmjTj17HkiCn4hbxWNGYKP51dQPeurMFDrjxpcX8deqX7NNolZyGilskSLS+sDSzhyRwZ9sLeGfpt3RI0miVnJqKWySIFIyO4KGu9fhgQDzFCkZx26vpDHlnOT8eOOx1NAkiKm6RINS4SnE+urcNQ9vV4uMvd9B+RKpGq+Q3Km6RIBUd6WNou9rMuLctVUoWyh6tStdolai4RYJdnfJF+OCe1jzc9SLSNv6g0SpRcYuEggifcUfbGswZmkD9SsU0WpXPqbhFQki1UrG8dWcLnuip0ar8TMUtEmLMjBubVyV5eCJtamaNVvUcu5ANOzValV+ouEVCVPliMbzUN47RNzbhuz0H6fbcfEbO+0qjVfmAilskhJkZ3RtVZN7wRLo0qMDIeV9z5XNprNBoVVhTcYuEgZKx0Yy6oQkv3xLH3l+P0vOFBfxz5lqNVoUpFbdIGLniov+MVr00fwsdR6aycNMPXscKS1OfXcbUZ5d58thnLG4zq2Jmn5nZOjNbY2ZD8iKYiJybnNGqt+9siRn0fmkJD36g0apw4s8Z9zHgPufcRUBLYKCZ1QtsLBE5X60uLMXsIQn0T6jBu198S/ukFOat1WhVODhjcTvndjjnlmX/eD+wDqgU6GAicv4KRkfw1y4XMXVAPCUKRXPH6+kMflujVaHOzuZls2ZWHUgF6jvn9p3uuLi4OJeenn7e4UQk9xw5doKxn29izGdfU7hAJI92v5jujSpiZl5HCwm/v579/ddZd+5UrFX8vz5+9X1Nz+nzm1mGcy7On2P9fnLSzAoD7wNDT1XaZtbfzNLNLD0zM9P/tCKSJ6IjfQxpV4uZg9tSrVQsQ95ZwR2vpbNj769eR5Oz5NcZt5lFATOAOc65pDMdrzNukeB2/IRj4oIt/HvuBiJ9Ph7sUpcbm1XF59PZt79yzsDP9Qz793L1jNuyvo56GVjnT2mLSPDLGa2aOzSRhpWL8dDU1fSesJhvftBoVSjw51JJPNAHuNzMVmS/dQlwLhHJA1VLFWLSHS14smcD1mzfR8eRqYxP3cSx43rZfDCLPNMBzrk0QF8/iYQpM+OG5lW5tE5ZHp62mn99vJ6Zq3bwVK+G1C1f1Ot4cgpndVeJv3SNWyQ0OeeYsWoHj05fw95fjzLgspoMvOxCCkRGeB0t7AXkrhIRCX9mxpWNKpI8PJErG1Vk9CdZo1XLv/3J62hyEhW3iPyPkrHRjLi+Ma/cGsf+Q8foOXYhf5+xloNHjnkdTVBxi8gfuLxuOeYOS+CmFlV5OW0LnUbOZ+FGjVZ5TcUtIn+oSEwU/7iqAe/0b4nPoPeEJfzl/VXs/VWjVV5RcYuIX1rWKMXsoQnclViD99K/o8OIFJI1WuUJFbeI+C0mKoIHO1/EtIFZo1V3vp7OoLeW8YNGq/KUiltEzlrDysWZPqgN97Wvzdw1u2iflMK05dsJxO3F8r9U3CJyTqIjfdx7RS1mDm5D9dKxDH13Bbe/ls73P2u0KtBU3CJyXmqVK8KUu1vzt271WLTpRzqMSOXNxVs5cUJn34Gi4haR8xbhM25rcwFzhibQqEoxHp62mhtfWswWjVYFhIpbRHJN1VKFePP2Fjx9TUPW7thHp5GpvJii0arcpuIWkVxlZlzXrArzhieSULsMT8xaT8+xC1m347TfNEvOkopbRAKiXNEYxve5hOd7N+X7n3/lyufSSJq7gcPHjnsdLeSpuEUkYMyMrg0rkDwske6NKjL60410G53GMo1WnRcVt4gEXInYaJKub8zEfs345fAxrhm7kMc/0mjVuVJxi0ieuaxOWeYMS+DmFtV4ZcEWOo5MZYFGq86ailtE8lSRmCj+flV93u3fkkifj5smLOGBKRqtOhsqbhHxRIsapZg1pC13J17IlGXbaJ+Uwtw1O72OFRJU3CLimZioCP7SuS7TBsRTqnAB+r+RwcC3lpG5X6NVf0TFLSKea1C5GNMHxfOnDrVJXrOL9iNSmLp8m0arTkPFLSJBISrCx6DLa/HxkDbUKB3LsHdX0u/VL9iu0ar/oeIWkaBSs2wRJt/dmv+7sh5LNu+hQ1IKb2i06r+ouEUk6ET4jH7xFzB3WAJNqpbgkWmruWH8YjZnHvA6WlBQcYtI0KpSshBv3N6cp3s1ZP3OfXQeNZ9xGq1ScYtIcDMzrovLGq26tE4Znpy1nqteWMDa7/PvaJWKW0RCQtmiMYy7+RJeuKkpO/ceovuYNJ7Np6NVZyxuM3vFzHab2eq8CCQicjpmRpcG2aNVjSvy3Kcb6To6jYyt+Wu0yp8z7leBTgHOISLitxKx0SRd15hX+zXj1yPH6TVuIY99tIZfDueP0arIMx3gnEs1s+qBjyIieW5iV/+O6zczsDnO0aXZo1VPz17PxAXfkLx2F0/0bEDbWmW8jhZQuXaN28z6m1m6maVnZmbm1qcVEflDhQtE8niP+rx3VyuiI3z0eXkp909Zyd6D4TtaZf68pDT7jHuGc66+P580Li7Opaenn18yEZGzdOjocUZ98jXjUzdTMjaav/eoT6f65b2O5Rczy3DOxflzrO4qEZGwERMVwQOd6vLhwHjKFC7A3W9mMHBS+I1WqbhFJOzUr1SMDwfF8+eOdUheu4t2SSm8nxE+o1X+3A74NrAIqGNm28zs9sDHEhE5P1ERPgZeVpOPh7SlZtnC3Dd5JbdODI/RKr+ucZ8tXeMWkWBy4oTj9UXf8PScDRjwQOe63NyiGj6feR3tN7rGLSJyEp/PuDX+AuYMTaBptRL87cM1XD9+EZtCdLRKxS0i+UaVkoV4/bbmPNOrIRt27qfzqPm88PnGkButUnGLSL5iZlwbV4V59yVyeZ2yPD17A1e9sIA13+/1OprfVNwiki+VLRLDuD6XMPampuzce5juYxbwzJz1HDoa/KNVKm4Rydc6N6jAvOEJXNW4Es9/tomuo+eTsXWP17H+kIpbRPK94oWiefa6Rrx2W3MOHT1Br3GLeHR68I5WqbhFRLIl1i7DnGEJ9G1ZjdcWfUOHEamkfhV820sqbhGRkxQuEMlj2aNVBaJ89H1lKX+aHFyjVSpuEZFTaFa9JB8PbsuASy9k6vLttBuRwuzVO7yOBai4RUROKyYqgvv/a7RqGfe8mcHu/Yc8zaXiFhE5g5NHqz5Zv5v2SalM8XC0SsUtIuKH30arBrelVtnC/Cl7tOrIsbx/1eUZv3WZiIj8R82yhXnvrla8uWQrmzN/IToy789/VdwiImfJ5zP6tqru3eN79sgiInJOVNwiIiFGxS0iEmJU3CIiIUbFLSISYlTcIiIhRsUtIhJiVNwiIiHGAvFaezPLBLae408vDfyQi3FySzDmCsZMEJy5gjETBGeuYMwEwZkrNzNVc86V8efAgBT3+TCzdOdcnNc5fi8YcwVjJgjOXMGYCYIzVzBmguDM5VUmXSoREQkxKm4RkRATjMU93usApxGMuYIxEwRnrmDMBMGZKxgzQXDm8iRT0F3jFhGRPxaMZ9wiIvIHgrK4zewZM1tvZqvMbKqZFfc6E4CZXWtma8zshJl5+uy2mXUysw1mttHM/uJllhxm9oqZ7Taz1V5nyWFmVczsMzNbl/17NyQIMsWY2VIzW5md6TGvM+UwswgzW25mM7zOksPMvjGzL81shZmle50nh5kVN7Mp2V21zsxa5dVjB2VxA8lAfedcQ+Ar4EGP8+RYDfQEUr0MYWYRwPNAZ6AecKOZ1fMyU7ZXgU5eh/idY8B9zrmLgJbAwCD4tToMXO6cawQ0BjqZWUuPM+UYAqzzOsQpXOacaxxktwOOAmY75+oCjcjDX7egLG7n3Fzn3LHsdxcDlb3Mk8M5t845t8HrHEBzYKNzbrNz7gjwDtDD40w451KBPV7nOJlzbodzbln2j/eT9ZerkseZnHPuQPa7Udlvnj/ZZGaVga7ABK+zBDszKwokAC8DOOeOOOd+zqvHD8ri/p3bgFlehwgylYDvTnp/Gx6XUSgws+pAE2CJt0l+uySxAtgNJDvnPM8EjATuB/L+u9/+MQfMNbMMM+vvdZhsNYBMYGL2paUJZhabVw/uWXGb2TwzW32Ktx4nHfMQWV/qTgqmXEHATvExz8/YgpmZFQbeB4Y65/Z5ncc5d9w515isryabm1l9L/OYWTdgt3Muw8scpxHvnGtK1qXBgWaW4HUgsr5fb1NgrHOuCfALkGfPNXn2zYKdc+3+6P+b2S1AN+AKl4f3LJ4pV5DYBlQ56f3KwPceZQl6ZhZFVmlPcs594HWekznnfjazz8l6bsDLJ3Xjge5m1gWIAYqa2ZvOuZs9zASAc+777P/uNrOpZF0q9PR5JrL+Dm476SulKeRhcQflpRIz6wQ8AHR3zh30Ok8Q+gKoZWYXmFk0cAMw3eNMQcnMjKzrkOucc0le5wEwszI5d0qZWUGgHbDey0zOuQedc5Wdc9XJ+vP0aTCUtpnFmlmRnB8DHfD2HzgAnHM7ge/MrE72h64A1ubV4wdlcQNjgCJAcvYtQOO8DgRgZleb2TagFTDTzOZ4kSP7idtBwByynmx7zzm3xossJzOzt4FFQB0z22Zmt3udiawzyT7A5dl/llZkn1V6qQLwmZmtIusf4WTnXNDcfhdkygFpZrYSWArMdM7N9jhTjnuBSdn3a8DgAAAAQ0lEQVS/j42Bf+XVA+uVkyIiISZYz7hFROQ0VNwiIiFGxS0iEmJU3CIiIUbFLSISYlTcIiIhRsUtIhJiVNwiIiHm/wGVde6UqmWnmQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fed4d0481d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# https://maviccprp.github.io/a-support-vector-machine-in-just-a-few-lines-of-python-code/\n",
    "for d, sample in enumerate(x):\n",
    "    # Plot the negative samples\n",
    "    if d < 2:\n",
    "        plt.scatter(sample[0], sample[1], s=120, marker='_', linewidths=2)\n",
    "    # Plot the positive samples\n",
    "    else:\n",
    "        plt.scatter(sample[0], sample[1], s=120, marker='+', linewidths=2)\n",
    "\n",
    "# Print a possible hyperplane, that is seperating the two classes.\n",
    "plt.plot([-2,6],[6,0.5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "ld = 1 / 1000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_samples, n_features = x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "w = np.zeros(n_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.,  0.,  0.])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nn(x, w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0,  1, -1])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sign([0, 10, -11])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-1, -1,  1,  1,  1])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nn(x, w)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.multiply(t, nn(x, w))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 1, 1, 1, 1])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.fmax(1 - np.multiply(t, nn(x, w)), 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.,  0.,  0.])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.,  0.,  0.])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w ** 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 1.,  1.,  1.,  1.,  1.])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.fmax(np.sign(1 - np.multiply(t, np.dot(x, w))), 0) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 7,  7, -1])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum(np.multiply(t.reshape(-1,1), x), 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-1, -1,  1,  1,  1])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-1, -1,  1,  1,  1])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t.T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ True,  True,  True,  True,  True], dtype=bool)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t == t.T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 7.,  7., -1.])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum(np.multiply(t.reshape(-1,1), x) * np.fmax(np.sign(1 - np.multiply(t, np.dot(x, w))), 0).reshape(-1,1), 0) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5.0"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sum(np.fmax(1 - np.multiply(t, nn(x, w)), 0).reshape(-1, 1) + np.sum(ld * w ** 2, 0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[-2,  4, -1],\n",
       "        [ 4,  1, -1],\n",
       "        [ 1,  6, -1],\n",
       "        [ 2,  4, -1],\n",
       "        [ 6,  2, -1]]), array([[-1],\n",
       "        [-1],\n",
       "        [ 1],\n",
       "        [ 1],\n",
       "        [ 1]]), array([[ 2, -4,  1],\n",
       "        [-4, -1,  1],\n",
       "        [ 1,  6, -1],\n",
       "        [ 2,  4, -1],\n",
       "        [ 6,  2, -1]]))"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x, t.reshape(-1, 1), x * t.reshape(-1, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.],\n",
       "       [ 1.],\n",
       "       [ 1.],\n",
       "       [ 1.],\n",
       "       [ 1.]])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.fmax(np.sign(1 - np.multiply(t, np.dot(x, w))), 0).reshape(-1,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cost(y, t, w, ld): return np.sum(is_misclassified(w, x, t)) + ld * np.sum(w ** 2, 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cost(y, t, w, ld): return np.sum(is_misclassified(w, x, t)) + ld * (np.linalg.norm(w, ord=2) ** 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def is_misclassified(w, x, t): return np.fmax(np.sign(1 - np.multiply(t, np.dot(x, w))), 0).reshape(-1,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def is_misclassified(w, x, t): return (np.multiply(t, np.dot(x, w)) < 1).astype(int).reshape(-1,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gradient(w, x, t, ld): return np.sum((2 * ld * w) - x * t.reshape(-1, 1) * is_misclassified(w, x, t), 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def delta_w(w, x, t, learning_rate, ld):\n",
    "    return learning_rate * gradient(w, x, t, ld)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAIABJREFUeJzt3Xl4XPV97/H3d7R4w4ssG4OxLVvUoWATwFJAkI0sEEMJkOUGTJqtITQkaZqmTwpJ+wB1c3MpXS5PGm4dJyxNLphASAihgOES1gQZLDBgGxwbgYwweBGyDTa2lvneP+bMeGY0m5ajkXQ+r+eRZ87vnDnndzTyfOe3m7sjIiICECt3BkREZORQUBARkRQFBRERSVFQEBGRFAUFERFJUVAQEZEUBQUREUlRUBARkRQFBRERSaksdwb6a8aMGT5//vxBnWPHWwfZvvdAzn2zpozn8MnjBnV+EZGRpqWlZZe7zyx23KgLCvPnz2ft2rWDOscta7byvV8/3ye9Iga3/eVpNNTVDOr8IiIjjZm1lXLcqAsKg9HS1klzawfbdr9DzCCeNe1Tbxw2vfGWgoKIRFZkgkJLWyef/WkzXT1xKmNGLGbEe/tOBnjD71/molPmlSGHIiLlF5mG5ubWDrp64sQdenqdGZOqcx73TlfPMOdMRGTkiExQaKqvpboyRgyIA9v3Hsx53HGzpw5rvkRERpLIBIWGuhpuvriJ9y6cgQH5VpH46gePHs5siYiMKJEJCpAIDGctPjJ/QPhAvRqZRSTSItPQnLRh254+aWbwl++v5/Kzjy1DjkRERo5IlRRa2jq5fe2rfXc4TJ5QNfwZEhEZYSIVFJpbO+jJHpxAon2hZmLu3kgiIlESqaCQ7IFUYVBhh9JjQOf+rrLlS0RkpIhUm0KyB1Jzawc1E6tZfvcGunviVFXGaKqvLXf2RETKLlJBAUj1Lrrj6XYW1E6ic38XTfW1NLd2ZOwXEYmi0IKCmd0AnAPscPfFBY57D9AMXODuvwwrP0ktbZ0sW/kEXWlTXNy5bhsGjKuKcfPFTQoMIhJZYbYp3AQsLXSAmVUA/wysDjEfGZpbO+jOMeeRA1098VSJQUQkikILCu7+KPBmkcP+CrgD2BFWPrI11ddSkeeuY2ZqWxCRSCtb7yMzOwr4BLBiOK/bUFfDBe/pOwtqRcxYft5iVR2JSKSVs0vqtcBl7t5b7EAzu8TM1prZ2p07dw76wotyTHp3wXvmaspsEYm8cvY+agRuNTOAGcDZZtbj7ndmH+juK4GVAI2NjfmmLioqfZGd9EnxYgYW7FdJQUSirGxBwd0XJJ+b2U3A3bkCwlBJX2QnZpn74g6rntzKHU+3c8U5i1LdVBUgRCRqwuySugo4HZhhZu3AlUAVgLsPazsCJHodHeyO4/RdhpMg7WB3nCt+s564O9WV6p4qItETWlBw92X9OPaLYeUjqWZidd4ps1P5AHrjjgPdQfdUBQURiZLIzH3Uub8LK35Yqn2hIqbuqSISPZEJConxCYXDQswgljzGSgkhIiJjS2SCQkNdDcvPW5wxO2q2uENPb6L6qKdXo5tFJHoiExQALjplHh85dlbBY5LtDnHXGgsiEj2RCgotbZ08vKm0GTW0xoKIRFGkgkK+ldfSJRfhqa7SGgsiEj2RWk+hqb6WyooY3T1xzPqOVzh5fg2XnXUsza0dGrwmIpEUqaAAgCciQXZ7c8zgsrOOpaGuRsFARCIrktVHuUY1X/L+egUDEYm8SJUUmuprqa5MVh9ZRvvChtf30tLWCaDqIxGJrEgFBYBPLpmDAZPHVbLi0dZU+uObd/HES7twDNfcRyISUZEJCumzpFbGjHm1kzL2O9ATTz47tDSngoKIRElk2hSaWzvo6okTd+jqdbbseLvg8VqaU0SiKDJBIdmeUOqMRqcdrTYFEYmeyASFhroabr64iWWnzKOy0ARIgfWv7RmGXImIjCyRCQqQCAyLZ0+ltzezP6oBMyZnznPUub871RtJRCQqIhMUWto6+d6vn+cf7ny+z2I7FTHj2x89JiPNgTuebh+2/ImIjASR6H2U7HmUXI4znQHLz1vMRafM4+FNO7h/4/aMfSIiURKJkkKy51GuqfBiMTjmiMm0tHXiJKa7AKiqMD65ZM5wZlNEpOwiUVJoqq+lMmZ09fYNC71xuPreF3j21d0Z+4ut5ywiMhZFoqTQUFfD/2icm3f/U6909gkYvb2ulddEJHIiERQgMb1FZT/u1tHKayISPZEJCgCxWOJ2K2LG+SfOLnwsWnlNRKInMkGhubWDnt54YsOdhbMm84NPHM/82ol9jjW08pqIRFNkgkJymosKS5QUtu1+B4CFsyb3OXbSuAquOGeRprkQkcgx99HVz6axsdHXrl07oNe2tHWy4pGX+N2LO4gHi+3kM75KU2eLyNhhZi3u3ljsuNBKCmZ2g5ntMLP1efZ/1syeC37+YGYnhJWXdL97cQe9RQICQHcwdbaISJSEWX10E7C0wP6XgQ+6+7uBfwJWhpgXAH71dDu92etwBiorjDOPm5WqYqqqVJuCiERPaIPX3P1RM5tfYP8f0jabgdCHD+9462CftJPn17Bw1uTU6GUHduw9wAXvmaeqIxGJnJEyovnLwL1hX+TwyeP6pD29dTeXnXUsAMtWPpEaxLbx9fUcc8RkBQYRiZSy9z4ysw+RCAqXFTjmEjNba2Zrd+7cOeBrfXLJnD5rKfTGEyOXm1s7MkY1d/c6v9IsqSISMWUNCmb2buCnwHnunrdV191XunujuzfOnDlzwNdrqKth+bmLOWra+EPnJjFyuam+luy1d0ZXvywRkcErW1Aws3nAr4DPufsfh+OaLW2dLL97A9t2HziUD+ChTTtobu3gK++vp8KCwWsVxqeWzOGWNVv53PVruGXN1uHIoohIWYXWpmBmq4DTgRlm1g5cCVQBuPsK4AqgFvg/ZgbQU0of2sHINYW2Aw9s3M4DG7dTGTM+fsJsOvZ1sejIKax45CUeCNZXeGzzLgAuOmVemFkUESmrMHsfLSuy/2Lg4rCun0vNxGrMDPIM2OuJO3eu24ZxKAik+8VTWxUURGRMK3tD83BpaevkqrvW5x2nkC7fEbOmjM+zR0RkbIhMUGhu7aA7xyI7/XH6MYcPUW5EREamyASFpvpaqrK7F6WpiMGZx83i5Pk1TKyu6LNfU2mLSBREJig01NVw1bmLyRUWLPj53aYdPPlKJ/u7evscU6lpL0QkAkbKiOZhsWHbnpztBU5ireZC0+TFR9lssiIiAxGZkgL0bUCOGVRXxohBzhJEuh6NcBaRCIhUUPhU2jQXBnz02Fn8xWnzAYiX8HqVFURkrItU9REcioIO3L9xOzHrGxCSpYZYLFGtlLR49tTwMygiUkaRKik0t3bQkzVOIdewBbNE0IinBYRCvY+uvucFTv+Xh7j6nheGLrMiImUQqZJCcp3mA92FK4uSgSIZL4zM3kctbZ00t3bQVF/LAxveYMWjrQCpx8vPPjaM7IuIhC5SQaGhroabL27KmNOoFM6h3kctbZ0s+0kz3T1xqipjjK/MLGzd1tKuoCAio1akggIkAsOJc6fx4Avbc1Yd5ZPe+6irJ556tKzm51ixbkwiIiNYpNoUkprqa6nM8ek9bUJVwdc5sHn7WxlpE6oyRz8vmVejNgYRGbUiGRSARGtylt3vdOc9vDJYX+FgT2Z7xKRxmYWtA929rHi0lVc69rPi0VYFBhEZVSIXFFraOrn2//2R7p5SRiYckvxFXfCezKmzF82emurCasDTWzsz9t+57jVAPZREZHSIVJtCS1snn/1pc9HeR7n0xp07nm7nqGkTOP/E2ax7dTdLFx3BvNpJ3B80WjtQO2kcbx/cn3rd3OkTufqeF9RDSURGhUgFhebWDg4OICAAWMz4ZUs73cHKbTGDm554hQ8szFwzOnuG1YZ5NanSQtKd615LBYWr73mB+za8wdJFRyhQiEjZRSooNNXXUhGzPgPYSuFxp9sPTZkXd+juifNMVnXR1jf3Z2xveH0vc6dP5I29B1Npc6dPBFAJQkRGnEi1KTTU1XDx+xakuo325+Z7ve/cRxUVMQ5ktU309GZuLzpyCg3zajLzEWznKkFkU1uEiAynSAWFlrZObnriFdyhMmZc8oH6orOjFhKPx5kzbUJG2pSsbq1vHezhidaOjLTkds3E6oz07O1kSaJYT6Zb1mzlc9ev4ZY1Wwd0HyIiSZEKCs2tHXQFbQJxd55o7RjUzKc9cThsfGYN3IzDxmVs73zrIIdnre2c3E5WIyVlb5dSkrhlzVa+9+vneWzzLr736+cVGERkUCIVFGomVqdGMccdxlUO/vY792eObeju7duQffSMSQW38xlXWVFwG+CG379ccDvdt259hhOX38+3bn2mpOuLSPQUbGg2s99SYBkBdz93yHMUos79XRiJG4oBU7OqawaiKmtkdFVFZqCZMXkcG17fm5GW3N6dNetq9nb2mtK51pjek/Wa7O2kb936DHeu2waQerz2wpNyHpv9uof/uJPT3zWzpOPD8vnr1/DkK29y8vzp/OzLp5QtHyJjXbHeR/8aPH4SOAL4v8H2MuCVkPIUmqb6WsZVxejuiVMRM3bsPTDoc751ILOksDdre/HsqUwZV8ljm3el0hYdOQWAN/dlfoBnb2cHmOxtALfC20mrN2wvuJ3LQAPJUEkGgvGVMXa/0wPAo5t38fnr1ygwiISkYFBw90cAzOyf3P0Dabt+a2aPhpqzECRnSb3j6XZ+2dLOc+17Bn3O3VnVR11ZvZEe3rSjTwkgOep5+qRq2LkvlT59UmbJJbsqKlfV1LisQJG9nRSLFd7OZSCBJN1gxmB8/vo1PBoE0uzBhn94qSPXS/rl/B89zvpte1k8ewp3fuN9gz5feklm7zvdQ3rukeqWNVu5d/3rnLX4SC46ZV7xF8ioUOo4hZlmVu/urQBmtgCYWeQ1I1JDXU1isZ3eeN56sZjlXnwnl7e7ejO2e+KZH2CtO9/m7YM9GWnZYxnyKRY0AKaMr+I1DmRs59KnmquU6VzdC28XMJAxGOkfMs2thT74B9494JY1W/mf/72RfcH7tq59D+f/6PFBfXinB7BH00qEQ3HupGQQW1A7kfOXzKGpvpaGupriLxwi77v6QV7bfYCjpo3n8cs/kurgAKRKweUODP39EjLUXwzGilJbWv8GeNjMHjazh4GHgG8VeoGZ3WBmO8xsfZ79ZmY/NLMtZvacmS3pV84H4qqpcNVUvvZIAy9VX8TL43L/FNoH5N0GeLr3MxlpVRUxnuj6VCoLL4+7iCOmjIerpvLangOp1wH84o2zMvOa9ppc6QD37P54we3k8e8E37aT53on/dv3VTmWGb1qap9lSjdWXNj3uDyvv+XJrRn5vuXJrYWPDz5kfrb1TL736+dzzmKbyn+OyQxznTu7q27yGuvtgoyXPP9ajhJjrt9JnuulVw1meza9NFrqObOOPf9Hj7OufQ89cWfzzn38y+pNfPanzbS0deY8vl9KeN37rn6Q9t0HcKB99wHed/WDLLvn+Ixjvn/3hiG9Zn9f//nr1/RrIkq/amrqd5oM3kOZn0G5ampZu5mXVFJw9/vMbCHwp0HSi+5+sNBrgJuAHwE/y7P/LGBh8HMK8J/BY+gs9c8AX2/5t5PPk4/dcc+4lBm8EbRlHOzuzXxt1nU273gr41zJ7YxrF9lOSi4SlDxXvIRv/dkD8Qp9Fmfb39WDVWduF/K/7tmYcY39eaYjMSs+xTmQ85vsNatfzLhG0tQSzldIod/kYMbBJL/5vtLRt2TZ3ROnubVjWEoL7bsP9Nm2zF7Wed+vwSj1m/8ta7ZmlNCg+GJX2e/LuiGoSi5VKfdVzlJYSSUFM5sIfAf4hrs/C8wzs3MKvcbdHwXeLHDIecDPPKEZmGZmR5aY71Fj+sS+HzjJQWrZq7Zl23ewt+B2f8SyPgmzt3MZwGwgKdnNHzmaQzK81Y97+/YZxxQ95hdPbe2zvXd/7qnRlx4f3p/duSfOHvBrk998c6lKWx52JMj1dz4YpQ7cBLjiN30rIyZUjcze9v25r6R+lcKGQKm/uRuBLuDUYLsd+P4gr30U8GradnuQ1oeZXWJma81s7c6dOwd52fCccdysPml/Mmtyn7STgm93x84uXOw8euZhBbf7Y8Zh1QW3c5kyYeBTY2W3d+dp/04p9od4/omzef/CGUBp35pyDRicnON+qoN1MobatIlVnH/i7FB6a42rjHHzxU3D2qZQzE++8J4hPd99G94ouJ0u11xmX//Qwn5db/K4vmOAwtCf+0oKoxRWSKlB4Wh3vwboBnD3dxhcyZg8r8/53dTdV7p7o7s3zpxZ3vbt7A+k6rRv+1/94NGJtLTxBMkPnOoKS91wMi15PECuQsP3P3F86g2KBdu5xLIec8n+T1LKf5rLlg58cr5z3j274Ha27G/U5584mx984njev3AGP/jE8Vx74Un8vB/dUL/6waNTv9PKWGI7+37OPG4Wqy45ddAfrnOmje+zve6KM0PrvnvlxxcNa0A4LGvm3+T2HZeexnc+dgx3XHrakOdn6aIjCm6nm5YV7CdUxvpd3XLTXwxPF+f+3FfSwpmlDXYdKqV+FewyswkEH9pmdjRQrE2hmHZgbtr2HGDbIM8Zuh984nh49tD2qq80JcpRkPqPseqSU3OmNbd2wCOH0pKP3/nYMYmqgBszr9VQV8Ptl54GN8LtBf7j3X7paTS3duQ8R1LqP8m9iXso5T9N8phkjyDuLfqSlGsvPAmuSnxjLmXgW2r/i2R8wx5oXWpDXQ2/+MtDv5eGuppDv79+/A5K8fjlH+nTO2cofPUD9am653m1k8rW/XP98qUsvuI+3u7q5bDqCtYvXwpXkfk7HWLJuvZS2hTWXfkxTvzH1ex+p4dpEypZd+XHSrrGHZdm/n0Mh1Lva+HMSby0ax9Hz5jEA397+rDkLanUoHAlcB8w18xuBt4LfHGQ174L+IaZ3UqigXmPu78+yHMOu1x/TPnSGupq4JG+5/j6h/6k6PkL/dGW+p/zolPmwb39+6C96JR5GQGlv9ZdcWbJxyYDyVB9w871exnI76AUQxUI0l1+9rEZHxrl7PK5fvnSYb9m9v0XUmogSBdmUCuklPsa7kCQzrzEvudmVgs0kaj2aXb3/P3wEsevAk4HZgDbSQSWKgB3X2FmRqJ30lJgP/Ald19bLB+NjY2+dm3Rw0REJI2Ztbh7Y7Hjis199Kfu/mLaGILkN/l5ZjbP3Z/O91p3X1bo3J6IRl8vlsGh1tLWOexFRhGR0aJY9dG3gUuAf8uxz4EPD3mOQpRco7mrJ071COzBISJSbsWCwgPB45eTU1yMZsn1FJJLaQ7X4B8RkdGiWJfU7waPvww7I8Ohqb6W6soYFTbyBv+IiIwExUoKHWb2ELDAzO7K3jna1lNIzpKqNgURkdyKBYU/A5YAPyd3u8KoU65uaCIio0Gx9RS6gGYzO83dR+78Ev2g3kciIvkV65J6rbt/C7jBzPoMaBht1UfqfSQiUlix6qOfB4//WvCoUUK9j0RECitWfdQSPKYmZzCzGmCuuz8Xct6GXLL3UXdPXL2PRERyKGnuo2C1tXOD49cBO83sEXf/doh5G3LqfSQiUlipE+JNdfe9ZnYxcKO7X2lmo66kAOp9JCJSSKlBoTJYFe0zwN+HmJ/QtbR1csfT7RjwySVzFCBERNKUGhSWA6uBx939KTOrBzaHl61wtLR1smzlE3T1JjpS3d7SzqqvqAeSiEhSSSuvufvt7v5ud/9asN3q7p8KN2tDr7m1g+7eQz1rkz2QREQkoaSgYGbXmNkUM6syswfNbJeZ/XnYmRtqTfW1VKUtlakeSCIimUpdo/lMd98LnENiGc13Ad8JLVchaairYdUlp3LGcbM4Yc5UrhrmtW5FREa6UtsUqoLHs4FV7v5mYuG00WfTG2/x0Is7iLuzafsGjjlisgKDiEig1JLCb83sRaAReNDMZgIHwstWOFraOrniN+vpiTtxhy61KYiIZCi1ofly4FSg0d27gX3AeWFmLAzNrR30xg81NMfM1KYgIpKm1OojgKOAM8xsfFraz4Y4P6Fqqq9lXFWMg91xzODi9y1Q1ZGISJpSex9dCfxH8PMh4BoS016MKg11NVxxziIqYon2kJueeIWWts7yZkpEZAQptU3h08BHgDfc/UvACcC40HIVos79XcTdM2ZKFRGRhFKDwjvuHgd6zGwKsAOoDy9b4amZWI0BBlTE1KYgIpKu1DaFtWY2DfgJ0AK8DTwZWq5C0tLWyVV3rSc5qLnPqkEiIhFXUlBITm8BrDCz+4Apo3E9hT7TXPS6FtoREUlTsPrIzJZk/wDTScyauqTYyc1sqZltMrMtZnZ5jv3zzOwhM3vGzJ4zs7MHfivFNdXXUpF1xzUTq8O8pIjIqFKspPBvBfY58OF8O82sArgOOIPE1BhPmdld7r4x7bB/AG5z9/80s+OAe4D5pWR8IBrqavjwn87i/o3bU2kbtu0J63IiIqNOseU4PzSIc58MbHH3VgAzu5XEgLf0oODAlOD5VGDbIK43IGpXEBE5pNRxCl8PGpqT2zVm9rVCryEx2O3VtO32IC3dVcCfm1k7iVLCX5WSn4Fqaevkdy9uz0hbPHtqmJcUERlVSu2S+hV3353ccPdO4CtFXpNrxrzsL+bLgJvcfQ6JyfZ+bmZ98mRml5jZWjNbu3PnzhKz3FdimovMtM79XQM+n4jIWFNqUIhZ2rSoQXtBsRbadmBu2vYc+lYPfRm4DcDdnwDGAzOyT+TuK9290d0bZ86cWWKW+1JDs4hIYaUGhdXAbWb2ETP7MLAKuK/Ia54CFprZAjOrBi4E7so6ZiuJkdKY2bEkgsLAiwJFJBua061XQ7OISEqpg9cuAy4BLiVRLXQ/8NNCL3D3HjP7BomAUgHc4O4bzGw5sNbd7wL+FviJmf0NiaqlL7r7sLb9js5VIUREwlHq4LU4sILE4LXpwBx37y3hdfeQaEBOT7si7flG4L39yvEg5GpoXqSGZhGRlFJ7Hz0crNE8HVgH3Ghm/x5u1oZec2sHPVkNzao+EhE5pNQ2hanBGs2fBG509wbgo+FlKxxN9bVUZNUXqfpIROSQUoNCpZkdCXwGuDvE/ISqoa6Gj58wOyNt8rj+rDMkIjK2lRoUlpNoMN7i7k+ZWT2wObxshWf9a5nVRU9oPQURkZRSG5pvB25P224FPhVWpsLS0tZJ6659GWmzpozPc7SISPQUDApm9nfufo2Z/Qc5pgly92+GlrMQNLd2EM+6i/oZk8qTGRGREahYSeGF4HFt2BkZDk31tcSA9A5IKx9r5YxFR2hNBRERis+S+tvg8b+GJzvhaqirof7ww9iy4+1UWtzRQjsiIoFi1UfZ01JkcPdzhzY74Zs9dXxGUADNfyQiklSs+uhUEtNfrwLWMMq79be0dfLY5l190jVTqohIQrGgcASJldOWARcB/w2scvcNYWcsDM2tHTkX1VFJQUQkoeA4BXfvdff73P0LQBOwBXjYzEJdDCcsuUY0g0oKIiJJRccpmNk44M9IlBbmAz8EfhVutsLRUFfDgpmHqU1BRCSPYg3N/wUsBu4F/tHd1w9LrkLS0tbJyzvf7pOukoKISEKxksLngH3Au4Bvpi++Bri7Twkxb0OuubWD3hyNCiopiIgkFBunUOrcSKNCU31tIpplpaukICKSMKY+9ItpqKvh/Qv7LAGtkoKISCBSQaGlrZPHt2icgohIPpEKCrkmxAOVFEREkiIVFPKNU9CSnCIiCZEKCg11NTknvtuy/a0y5EZEZOSJVFAAeG33O33SDvbEcxwpIhI9kQoKLW2dbNt9oE/6Ai20IyICRCwo5JsQr2Ofeh+JiEDEgkJy8Fq22knqfSQiAhELCvkGr6mkICKSEGpQMLOlZrbJzLaY2eV5jvmMmW00sw1mdkuY+Wlp6+T3OQavqaQgIpJQdOrsgTKzCuA6Eov0tANPmdld7r4x7ZiFwHeB97p7p5kdHlZ+IP+EeC/v2hfmZUVERo0wSwonA1vcvdXdu4BbgfOyjvkKcJ27dwK4+44Q80NTfW3O9HGVkapFExHJK8xPw6NIrO+c1B6kpXsX8C4z+72ZNZvZ0hDzQ0NdDccdOblPeq4eSSIiURRa9RHk7OiT/flbCSwETgfmAI+Z2WJ3351xIrNLgEsA5s2bN+AMtbR18uLrfUcvd2nwmogIEG5JoR2Ym7Y9B9iW45jfuHu3u78MbCIRJDK4+0p3b3T3xpkzZw44Q82tHeT6+J86oWrA5xQRGUvCDApPAQvNbIGZVQMXAndlHXMn8CEAM5tBojqpNawM5WtTeGzLLlraOsO6rIjIqBFaUHD3HuAbwGrgBeA2d99gZsvN7NzgsNVAh5ltBB4CvuPuHWHlCfLUaXmiFCEiEnVhting7vcA92SlXZH23IFvBz+hyzfNBWhNBRERiNiI5qb6WmK5igpoTQUREYhYUGioq+GYWX27pELuaiURkaiJVFBoaevkxTdyL6izaPbUYc6NiMjIE6mgUKhNQdVHIiIRCwr5uqQC7Hrr4DDmRERkZIpUUID8bQczJ48b1nyIiIxEkQoKhaqP1KYgIhKxoJBv5TVQm4KICEQsKDTU1XDCnNwlAnVJFRGJWFBoaevk2fbcJQJVH4mIRCwoqEuqiEhhkQoK6pIqIlJYpIKCiIgUFqmgUGh67O17D2hNBRGJvEgFhUJdUp9t38OylU8oMIhIpEUqKEDfRaLTdfU6dzzdPmx5EREZaSIVFEpZXU0NziISZZEKCoV6HyXt3t81DDkRERmZIhUUSvHmPgUFEYmuSAWFUqqPpk/SWs0iEl2RCgqlVB9Nm6igICLRFamgUAq1KYhIlEUqKJRSfaQ2BRGJskgFhZoSqobUpiAiURapoNCpqiERkYIiFRRKaWhW9ZGIRFmoQcHMlprZJjPbYmaXFzju02bmZtYYZn5KUVURqTgpIpIhtE9AM6sArgPOAo4DlpnZcTmOmwx8E1gTVl6SSmlo7u6Nh50NEZERK8yvxScDW9y91d27gFuB83Ic90/ANcCBEPMClFYHlTgMAAAIbUlEQVR9pJKCiERZmJ+ARwGvpm23B2kpZnYSMNfd7w4xH/2ikoKIRFmYQSHX0gWpmavNLAb8b+Bvi57I7BIzW2tma3fu3DngDJVSfaSSgohEWZifgO3A3LTtOcC2tO3JwGLgYTN7BWgC7srV2OzuK9290d0bZ86cOeAMlTJOQSUFEYmyMIPCU8BCM1tgZtXAhcBdyZ3uvsfdZ7j7fHefDzQD57r72rAyVMo4BZUURCTKQvsEdPce4BvAauAF4DZ332Bmy83s3LCuW4hKCiIihVWGeXJ3vwe4JyvtijzHnh5mXkAlBRGRYiL1CVhKl9S9B7q57qEttLR1DkOORERGlkgFhVK8tvsA/7J6Exf8+A8KDCISOZEKCqV0SU3qicOKR14KMTciIiNPpIJCKQ3N6Z7ZqpKCiERLpIJCf6fO3t/VG1JORERGpkgFhf6WFCL1yxERIWKfe/0tKWjEgohETaSCgkoKIiKFRepzr78lhe64Fz9IRGQMiVRQ6G9JIe4KCiISLZEKCv0tKcQs1+zfIiJjV6SCQn9LClUxBQURiZZIBQW1KYiIFBapoKA2BRGRwiIVFNZv21PuLIiIjGiRCgpqIRARKSxSQWHR7KnlzoKIyIgWqaDQ34ZmEZGoiVRQUEOziEhhkQoK/W1oVkwQkaiJVFDob0OzGqZFJGrMR9nXYTPbCbQN5LWxiVPnV06ZWVvyC9y9a/tLTw/kWiPIDGBXuTMxzHTP0aB77p86d59Z7KBRFxQGw8zWuntjufMxnHTP0aB7jobhuOdIVR+JiEhhCgoiIpIStaCwstwZKAPdczTonqMh9HuOVJuCiIgUFrWSgoiIFBCZoGBmS81sk5ltMbPLy52f/jCzuWb2kJm9YGYbzOyvg/TpZvaAmW0OHmuCdDOzHwb3+pyZLUk71xeC4zeb2RfS0hvM7PngNT80GxnLzplZhZk9Y2Z3B9sLzGxNkP9fmFl1kD4u2N4S7J+fdo7vBumbzOxjaekj7m/CzKaZ2S/N7MXg/T51rL/PZvY3wd/1ejNbZWbjx9r7bGY3mNkOM1uflhb6+5rvGgW5+5j/ASqAl4B6oBp4Fjiu3PnqR/6PBJYEzycDfwSOA64BLg/SLwf+OXh+NnAvifF3TcCaIH060Bo81gTPa4J9TwKnBq+5Fzir3Pcd5OvbwC3A3cH2bcCFwfMVwKXB868BK4LnFwK/CJ4fF7zf44AFwd9BxUj9mwD+C7g4eF4NTBvL7zNwFPAyMCHt/f3iWHufgQ8AS4D1aWmhv6/5rlEwr+X+TzBMb8ipwOq07e8C3y13vgZxP78BzgA2AUcGaUcCm4LnPwaWpR2/Kdi/DPhxWvqPg7QjgRfT0jOOK+N9zgEeBD4M3B38we8CKrPfV2A1cGrwvDI4zrLf6+RxI/FvApgSfEBaVvqYfZ9JBIVXgw+6yuB9/thYfJ+B+WQGhdDf13zXKPQTleqj5B9eUnuQNuoExeWTgDXALHd/HSB4PDw4LN/9Fkpvz5FebtcCfwfEg+1aYLe79wTb6flM3Vuwf09wfH9/F+VUD+wEbgyqzH5qZpMYw++zu78G/CuwFXidxPvWwth+n5OG433Nd428ohIUctWbjrpuV2Z2GHAH8C1331vo0BxpPoD0sjGzc4Ad7t6SnpzjUC+yb9TcM4lvvkuA/3T3k4B9JIr8+Yz6ew7quM8jUeUzG5gEnJXj0LH0PhdT1nuMSlBoB+ambc8BtpUpLwNiZlUkAsLN7v6rIHm7mR0Z7D8S2BGk57vfQulzcqSX03uBc83sFeBWElVI1wLTzKwyOCY9n6l7C/ZPBd6k/7+LcmoH2t19TbD9SxJBYiy/zx8FXnb3ne7eDfwKOI2x/T4nDcf7mu8aeUUlKDwFLAx6NFSTaKC6q8x5KlnQk+B64AV3//e0XXcByR4IXyDR1pBM/3zQi6EJ2BMUHVcDZ5pZTfAN7UwS9a2vA2+ZWVNwrc+nnass3P277j7H3eeTeL9+5+6fBR4CPh0cln3Pyd/Fp4PjPUi/MOi1sgBYSKJRbsT9Tbj7G8CrZnZMkPQRYCNj+H0mUW3UZGYTgzwl73nMvs9phuN9zXeN/MrZyDTMjTxnk+i18xLw9+XOTz/z/j4SxcHngHXBz9kk6lIfBDYHj9OD4w24LrjX54HGtHP9BbAl+PlSWnojsD54zY/Iauws8/2fzqHeR/Uk/rNvAW4HxgXp44PtLcH++rTX/31wX5tI620zEv8mgBOBtcF7fSeJXiZj+n0G/hF4McjXz0n0IBpT7zOwikSbSTeJb/ZfHo73Nd81Cv1oRLOIiKREpfpIRERKoKAgIiIpCgoiIpKioCAiIikKCiIikqKgIJLFzHrNbF3az5DNrGlm89NnyhQZaSqLHyISOe+4+4nlzoRIOaikIFIiM3vFzP7ZzJ4Mfv4kSK8zsweDue8fNLN5QfosM/u1mT0b/JwWnKrCzH5iiTUE7jezCWW7KZEsCgoifU3Iqj66IG3fXnc/mcSo0WuDtB8BP3P3dwM3Az8M0n8IPOLuJ5CYw2hDkL4QuM7dFwG7gU+FfD8iJdOIZpEsZva2ux+WI/0V4MPu3hpMUPiGu9ea2S4Sc9Z3B+mvu/sMM9sJzHH3g2nnmA884O4Lg+3LgCp3/374dyZSnEoKIv3jeZ7nOyaXg2nPe1HbnowgCgoi/XNB2uMTwfM/kJh9E+CzwOPB8weBSyG11vSU4cqkyEDpG4pIXxPMbF3a9n3unuyWOs7M1pD4QrUsSPsmcIOZfYfEymlfCtL/GlhpZl8mUSK4lMRMmSIjltoUREoUtCk0uvuucudFJCyqPhIRkRSVFEREJEUlBRERSVFQEBGRFAUFERFJUVAQEZEUBQUREUlRUBARkZT/D418YkwNKJGXAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fed4cbaeda0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "(0,\n",
       " 0.0042060248537924309,\n",
       " array([  1.58876117,   3.17458055,  11.11863105]),\n",
       " array([  3.19010824,   3.70734226,  19.91675431]))"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learning_rate = 1; nb_of_iterations = 100000\n",
    "\n",
    "w = np.zeros(n_features)\n",
    "costs = []\n",
    "costs_mis = []\n",
    "w_ref = np.zeros(n_features)\n",
    "cost_ref = 0\n",
    "costs_ref = []\n",
    "not_sames = []\n",
    "\n",
    "w_cost = [(w, np.zeros(n_features), cost(nn(x, w), t, w, ld))]\n",
    "\n",
    "for epoch in range(1, nb_of_iterations):\n",
    "    ld = 1 / epoch\n",
    "    \n",
    "    # print((- learning_rate * ((2 * ld * w) - x * t.reshape(-1, 1) * is_misclassified(w, x, t))))\n",
    "    \n",
    "    w = w - delta_w(w, x, t, learning_rate, ld)\n",
    "    w_cost.append((w, is_misclassified(w, x, t), delta_w(w, x, t, learning_rate, ld), cost(nn(x, w), t, w, ld)))\n",
    "    costs_mis.append(np.max(is_misclassified(w, x, t)) - 0.5)\n",
    "    costs.append(cost(nn(x, w), t, w, ld))\n",
    "\n",
    "    mis = is_misclassified(w_ref, x, t).reshape(-1, )\n",
    "    cost_ref = 0\n",
    "    not_same = 0.0\n",
    "    delta_w_ref_sum = []\n",
    "    for i, x_ in enumerate(x):  \n",
    "        if (t[i] * np.dot(x[i], w_ref)) < 1:\n",
    "            delta_w_ref = learning_rate * ( (x[i] * t[i]) + (-2 * ld * w_ref) )\n",
    "            cost_ref = 1\n",
    "            if mis[i] != 1:\n",
    "                not_same = 1.3\n",
    "        else:\n",
    "            delta_w_ref = learning_rate * (-2 * ld * w_ref)\n",
    "            if mis[i] != 0:\n",
    "                not_same = 1.3\n",
    "                \n",
    "        w_ref = w_ref + delta_w_ref\n",
    "        delta_w_ref_sum.append(delta_w_ref)\n",
    "        # print(delta_w_ref)\n",
    "    # print(delta_w_ref_sum)        \n",
    "                \n",
    "    costs_ref.append(cost_ref)\n",
    "    not_sames.append(not_same)\n",
    "    \n",
    "# plt.plot(costs_mis, 'o')\n",
    "plt.plot(costs, '.')\n",
    "plt.plot(costs_ref, '|') \n",
    "# plt.plot(not_sames, 's')\n",
    "plt.ylim(0.25, 1.5)\n",
    "# plt.axes().set_yticklabels([])\n",
    "plt.xlabel('Epoch')\n",
    "plt.ylabel('Misclassified')\n",
    "plt.show()\n",
    "\n",
    "costs_ref[-1], costs[-1], w_ref, w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.,  0.,  0.,  0.,  0.])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mis_raw = np.fmax(np.sign(1 - np.multiply(t, np.dot(x, w))), 0)\n",
    "mis_raw"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 2, -4,  1],\n",
       "       [-4, -1,  1],\n",
       "       [ 1,  6, -1],\n",
       "       [ 2,  4, -1],\n",
       "       [ 6,  2, -1]])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x * t.reshape(-1, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([ 0.,  0.,  0.,  0.,  0.]), array([[-2,  4, -1],\n",
       "        [ 4,  1, -1],\n",
       "        [ 1,  6, -1],\n",
       "        [ 2,  4, -1],\n",
       "        [ 6,  2, -1]]), array([-1, -1,  1,  1,  1]))"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mis_raw, x, t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0., -0.,  0.],\n",
       "       [-0., -0.,  0.],\n",
       "       [ 0.,  0., -0.],\n",
       "       [ 0.,  0., -0.],\n",
       "       [ 0.,  0., -0.]])"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x * t.reshape(-1, 1) * mis_raw.reshape(-1, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5,)"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mis_raw.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.,  0.,  0.,  0.,  0.])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mis_raw.reshape(-1, 1).reshape(-1, )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(np.multiply(t, np.dot(x, w)) < 1).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.fmax(np.sign(1 - np.multiply(t, np.dot(x, w))), 0).reshape(-1,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x * t.reshape(-1, 1) * is_misclassified(w, x, t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = (2 * ld * w) - x * t.reshape(-1, 1) * is_misclassified(w, x, t)\n",
    "b = np.sum(a, 0)\n",
    "a, b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cost(nn(x, w), t, w, ld)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ld * w ** 2, 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "w, np.sum(w ** 2, 0), np.linalg.norm(w, ord=2) ** 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def nn(x, w): return np.sign(np.dot(x, w)).astype(int)\n",
    "x = np.array([[-2,4,-1], [4,1,-1], [1, 6, -1], [2, 4, -1], [6, 2, -1]])\n",
    "t = np.array([-1, -1, 1, 1, 1])\n",
    "def cost(y, t, w, ld): return np.sum(is_misclassified(w, x, t)) + ld * (np.linalg.norm(w, ord=2) ** 2)\n",
    "def is_misclassified(w, x, t): return (np.multiply(t, np.dot(x, w)) < 1).astype(int).reshape(-1,1)\n",
    "def gradient(w, x, t, ld): return np.sum((2 * ld * w) - x * t.reshape(-1, 1) * is_misclassified(w, x, t), 0)\n",
    "def delta_w(w, x, t, learning_rate, ld): return learning_rate * gradient(w, x, t, ld)\n",
    "\n",
    "learning_rate = 1; nb_of_iterations = 100000\n",
    "\n",
    "w = np.zeros(n_features)\n",
    "costs = []\n",
    "\n",
    "for epoch in range(1, nb_of_iterations):\n",
    "    ld = 1 / epoch    \n",
    "    w = w - delta_w(w, x, t, learning_rate, ld)\n",
    "    costs.append(cost(nn(x, w), t, w, ld))\n",
    "\n",
    "w, costs[:5], costs[-5:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([  1.58876117,   3.17458055,  11.11863105]),\n",
       " [1, 1, 1, 0, 0],\n",
       " [0, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def nn(x, w): return np.sign(np.dot(x, w)).astype(int)\n",
    "def cost(y, t, w, ld): return is_misclassified(w, x, t) + ld * (np.linalg.norm(w, ord=2) ** 2)\n",
    "def is_misclassified(w, x, t): return (t * np.dot(x, w) < 1).astype(int)\n",
    "def gradient(w, x, t, ld): return 2 * ld * w - x * t * is_misclassified(w, x, t)\n",
    "\n",
    "x = np.array([[-2,4,-1], [4,1,-1], [1, 6, -1], [2, 4, -1], [6, 2, -1]])\n",
    "t = np.array([-1, -1, 1, 1, 1])\n",
    "learning_rate = 1; nb_of_iterations = 100000; w = np.zeros(n_features); costs = []\n",
    "\n",
    "for epoch in range(1, nb_of_iterations):\n",
    "    ld = 1 / epoch\n",
    "    error = 0\n",
    "    for i, x_ in enumerate(x):              \n",
    "        w = w - learning_rate * gradient(w, x[i], t[i], ld)\n",
    "        # costs.append(cost(nn(x, w), t, w, ld))\n",
    "        if is_misclassified(w, x[i], t[i]):\n",
    "            error = 1\n",
    "    costs.append(error)\n",
    "\n",
    "w, costs[:5], costs[-5:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = np.array([[-2,4,-1], [4,1,-1], [1, 6, -1], [2, 4, -1], [6, 2, -1]])\n",
    "t = np.array([-1, -1, 1, 1, 1])\n",
    "learning_rate = 1; nb_of_iterations = 100000; n_samples, n_features = x.shape; w = np.zeros(n_features); costs = []\n",
    "\n",
    "def nn(x, w): return np.sign(np.dot(x, w)).astype(int)\n",
    "def cost(y, t, w, ld): return max(1 - t * np.dot(x, w), 0) + ld * (np.linalg.norm(w, ord=2) ** 2)\n",
    "def is_misclassified(w, x, t): return (t * np.dot(x, w) < 1).astype(int)\n",
    "def gradient(w, x, t, ld): return 2 * ld * w - x * t * is_misclassified(w, x, t)\n",
    "\n",
    "for epoch in range(1, nb_of_iterations):\n",
    "    ld = 1 / epoch\n",
    "    error = 0\n",
    "    for i, x_ in enumerate(x):              \n",
    "        w = w - learning_rate * gradient(w, x[i], t[i], ld)\n",
    "        # costs.append(cost(nn(x, w), t, w, ld))\n",
    "        if is_misclassified(w, x[i], t[i]):\n",
    "            error = 1\n",
    "    costs.append(error)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([  1.58876117,   3.17458055,  11.11863105]),\n",
       " [1, 1, 1, 0, 0],\n",
       " [0, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w, costs[:5], costs[-5:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([  1.58876117,   3.17458055,  11.11863105]),\n",
       " [125.00000000000001,\n",
       "  20.5,\n",
       "  0.13771048902888564,\n",
       "  0.33270277832684497,\n",
       "  0.47185560838000012],\n",
       " [0.0013634192550939879,\n",
       "  0.0013631329528662499,\n",
       "  0.0013628467136208933,\n",
       "  0.0013625605373434335,\n",
       "  0.0013622744240193879])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = np.array([[-2,4,-1], [4,1,-1], [1, 6, -1], [2, 4, -1], [6, 2, -1]])\n",
    "t = np.array([-1, -1, 1, 1, 1])\n",
    "learning_rate = 1; nb_of_iterations = 100000; n_samples, n_features = x.shape; w = np.zeros(n_features)\n",
    "\n",
    "def nn(x, w): return np.sign(np.dot(x, w)).astype(int)\n",
    "def cost(w, x, t, ld): return max(1 - t * np.dot(x, w), 0) + ld * (np.linalg.norm(w, ord=2) ** 2)\n",
    "def is_misclassified(w, x, t): return (t * np.dot(x, w) < 1).astype(int)\n",
    "def gradient(w, x, t, ld): return 2 * ld * w - x * t * is_misclassified(w, x, t)\n",
    "\n",
    "costs = []\n",
    "for epoch in range(1, nb_of_iterations):\n",
    "    ld = 1 / epoch; error = 0\n",
    "    for i, x_ in enumerate(x):              \n",
    "        w = w - learning_rate * gradient(w, x[i], t[i], ld)\n",
    "        error = cost(w, x[i], t[i], ld)\n",
    "    costs.append(error)\n",
    "\n",
    "w, costs[:5], costs[-5:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEKCAYAAAAFJbKyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvAOZPmwAAIABJREFUeJzt3X2UXHWd5/H3t7tpRI0Ym6BISJo2kSP4gKQHGnQddkaFGDKoMEhkFTSAsOvMOHocAzg4OozGZZyDHLPEiCw6G2EcEQQSzDAsyriQmDSDJjFGYptAC5oQAkYDJN393T/qVlKprvtUVbdu9e3P65w+6XvrPvyqq3M//Xu619wdERGRpDryLoCIiEwsCg4REUlFwSEiIqkoOEREJBUFh4iIpKLgEBGRVBQcIiKSioJDRERSUXCIiEgqXXkXIAtHHHGE9/b25l0MEZEJZXBw8Cl3nxa3XSGDo7e3l3Xr1uVdDBGRCcXMtiXZTk1VIiKSSqGCw8zmm9myZ599Nu+iiIgUVqGCw93vcvdLDz/88LyLIiJSWIUKDhERyZ6CQ0REUlFwiIhIKgqOCoPbdrHk/i0MbtuVd1FERNpWIedx1GNw2y4uuHE1e0fG6O7qYPnFA8yZOTXvYomItJ22r3GY2bvN7Gtm9j0ze2dW51k9tJO9I2OMOewbGWP10M6sTiUiMqHlEhxmdpOZbTezDVXrzzSzzWa2xcwWAbj7He5+CXAR8L6syjTQ10N3VwedBod0dTDQ15PVqUREJrS8mqpuBr4CfLO8wsw6gSXAO4BhYK2Z3enuPws2+XTweibmzJzK8osHWD20k4G+HjVTiYiEyCU43P0BM+utWn0ysMXdhwDM7FbgbDPbBCwG7nH3h7Ms15yZUxUYIiIx2qmP42jg8Yrl4WDdXwBvB841s8vCdjazS81snZmt27FjR7YlFRGZxNppVJXVWOfufj1wfdzO7r4MWAbQ39/vTS6biIgE2qnGMQwcU7E8HXgizQF0k0MRkey1U3CsBWab2bFm1g2cD9yZc5lERKRKXsNxbwEeAo4zs2EzW+juI8BHgVXAJuDb7r4xzXF1d1wRkezlNapqQcj6lcDKFhdHRERSaKemqoapj0NEJHuFCg41VYmIZK9QwaEah4hI9goVHKpxiIhkr1DBISIi2StUcKipSkQke4UKDjVViYhkr1DBISIi2VNwiIhIKoUKDvVxiIhkr1DBoT4OEZHsFSo4REQkewoOERFJpVDBoT4OEZHsFSo41MchIpK9QgWHiIhkT8EhIiKpKDgqDG7bxZL7tzC4bVfeRRERaVu5PDq2HQ1u28UFN65m78gY3V0dLL94gDkzp+ZdLBGRtqMaR2D10E72jowx5rBvZIzVQzvzLpKISFsqVHA0Mhx3oK+H7q4OOg0O6epgoK8ngxKKiEx85u55l6Hp+vv7fd26dan3G9y2i9VDOxno61EzlYhMOmY26O79cdupj6PCnJlTFRgiIjEK1VQlIiLZU3CIiEgqCg4REUlFwSEiIqm0fXCYWZ+Zfd3MvpN3WUREJKfgMLObzGy7mW2oWn+mmW02sy1mtgjA3YfcfWEe5RQRkfHyqnHcDJxZucLMOoElwFzgeGCBmR3f+qKJiEiUXILD3R8Anq5afTKwJahh7AVuBc5ueeFERCRSO/VxHA08XrE8DBxtZj1mthR4s5ldEbazmV1qZuvMbN2OHTuyLquIyKTVTjPHrcY6d/edwGVxO7v7MjN7Epjf3d09p+mlExERoL1qHMPAMRXL04En0hxAj44VEcleOwXHWmC2mR1rZt3A+cCdaQ7QyN1xRUQkmbyG494CPAQcZ2bDZrbQ3UeAjwKrgE3At919Y5rjqsYhIpK9XPo43H1ByPqVwMp6j2tm84H5s2bNqvcQIiISo52aqhrWaI1DzxwXEYnXTqOqcqVnjouIJFOoGkcjneN65riISDKFCo5Gmqr0zHERkWTUVBWYM3MqV591AvdseJK5rz9KzVQiIiEKFRyNjKoa3LaLz929kb0jY6zd+jTHvWqKwkNEpAY1VQXUxyEikkyhgqMR6uMQEUlGTVWBOTOnsvziAVYP7WSgr0fNVCIiIQpV49AtR0REsleoGkcjNAFQRCSZQtU4GqHOcRGRZBQcgYG+Hro6DAM6O0yd4yIiIQoVHI0+j2PUHQ/+FRGR2goVHI10ji/94S8ZHSt9PzpWWhYRkfEKFRyN+NWO30cui4hIiYIj0DftpZHLIiJSouAIfOSPX0Nn8NPo7Cgti4jIeAqOCmZ20L8iIjKegiNw28PDjIyWRlONjDq3PTycc4lERNpToYKjkeG4T+1+IXJZRERKChUculeViEj2ChUcjThiyqGRyyIiUqLgCOx5YeSg5Z/9ur7Z5yIiRafgCNzxyBMHLT8y/CzfWvNYTqUREWlfCo4IS36wJe8iiIi0HQVHhF1/2Jt3EURE2k7bP8jJzF4C/C9gL/ADd1/eqnMrVUVExsvl2mhmN5nZdjPbULX+TDPbbGZbzGxRsPq9wHfc/RLgz1pZzt/vHW3l6UREJoTY4DCzDjN7s5nNM7M/MbNXNuG8NwNnVp2nE1gCzAWOBxaY2fHAdODxYDNdyUVEchbaVGVmrwE+BbwdeBTYAbwIeK2Z7QG+CnzD3cfSntTdHzCz3qrVJwNb3H0oOP+twNnAMKXweAS1HomI5C6qj+Ma4AbgI+4HPxLPzI4E3g98APhGk8pyNAdqFlAKjFOA64GvmNk84K6wnc3sUuBSgBkzZjSpSCIiUi00ONx9QcRr24HrmlyWWrekdXf/A/ChuJ3dfZmZPQnM7+7untPksomISCB2VJWZvbfG6meB9UGANMswcEzF8nTgiZBta3L3u4C7+vv7L2liuUREpEKS4bgLgVOB+4Pl04HVlPo6Pufu/9yksqwFZpvZscCvgfMpNYclZmbzgfmzZs1qUpFERKRaks7mMeB17n6Ou59DacTTC5T6Hz5Vz0nN7BbgIeA4Mxs2s4XuPgJ8FFgFbAK+7e4b0xxXd8cVEclekhpHr7v/tmJ5O/Bad3/azPbVc9Kw/hN3XwmsrOeYoBqHiEgrJKlx/IeZ3W1mF5rZhcCdwAPBjO5nsi1eOqpxiIhkL0mN439Qmr39Vkojn74B3BYM0f2vGZatpbo7jb2jHr+hiMgkFxsc7u5m9iNK94py4MfV8zraRSNNVd2dHewd1cR0EZE4SW45ch7wY+Bc4DxgjZmdm3XB6tFIU9UfdF8qEZFEkjRVXQX8UXnOhplNA/4d+E6WBRMRkfaUpHO8o2qi386E+7Wcmc03s2XPPpv+sa+1pq2LiMh4SQLg+2a2yswuMrOLgBU0MGQ2S400VR3W3ZlBiUREiic2ONz9k8Ay4I3Am4Bl7l7XxL92dsKrX5Z3EUREJoRETwB099uA2zIuS8MaGVU165VT+PHWXc0vlIhIwYTWOMxst5n9rsbXbjP7XSsLmVQjTVWvf3XtfRav3NRosURECiU0ONx9iru/rMbXFHcvXLvOxidqd6gvX7OtxSUREWlvUTWOl8btnGSbiWL77hdqrn9un+Z3iIhUiuoc/56ZfcnM3hbclwoAM+szs4Vmtoqq54ZPZEdOObTm+pHUD8YVESm2qKaqPwXuAz4CbDSzZ81sJ/B/gFcBF7p7W00CbGQex3tPmp5BiUREiidyVFWjtzlvtUaeADhn5tQMSiQiUjxtOQM8D4PbNBRXRCQJBUdg9dDO0NcUKiIiByg4AlNf3B362ld/+MsWlkREpL0lua36PydZN9Ht2rM39LUHtzzVwpKIiLS3JDWOEyoXzKwTmJNNcRrTyKiq3c+FPz7993pWh4jIflETAK8ws93AGytvNwJsB77XshKm0MgtRzY+2ZZ3URERaTtR8zi+4O5TgGurbjfS4+5XtLCMLTH39UdFvq4OchGRkiRNVXeXZ46b2X8zs38ys5kZl6vl3n/KjMjXv3iPbnYoIgLJguMGYI+ZvQn4G2Ab8M1MS9WG1qnGISICJAuOEXd34Gzgy+7+ZWBKtsVqvbimqDFXc5WICCQLjt1mdgXwAWBFMKrqkGyL1XpREwDL1FwlIpIsON4HvAB82N1/AxwNXJtpqXIw0NcTu42aq0REkj1z/DfAcuBwMzsLeN7dW9bHEdzG/etmlumdeJPc5FDNVSIiyWaOnwf8GPhz4DxgjZmdm+TgZnaTmW03sw1V6880s81mtsXMFkUdw92H3H1hkvM1qqvDYre57eHhFpRERKR9JWmqugr4I3e/0N0/CJwM/G3C499M1cOegj6SJcBc4HhggZkdb2ZvMLO7q76OTPxOmmD2kfEPNLxtUMEhIpNbkuDocPftFcs7E+6Huz8APF21+mRgS1CT2AvcCpzt7uvd/ayqr+3jDpqha97zhthtXhgZU3OViExqSQLg+2a2yswuMrOLgBXAPQ2c82jg8Yrl4WBdTWbWY2ZLgTcHo7vCtrvUzNaZ2bodO3bUVbCkD3NaqrvlisgkFvkEQAB3/6SZvRd4K2DAMne/vYFz1upI8Ijz7wQuizuouy8zsyeB+d3d3ZnehPHen/2Wb615LHa2uYhIEUXd5HCWmb0FwN2/6+4fd/e/Bnaa2WsaOOcwcEzF8nTgiQaOt18jNzlM6+rvbVCTlYhMSlFNVdcBu2us3xO8Vq+1wGwzO9bMuoHzgTsbON5+jdxWPa2RMU80aVBEpGiigqPX3X9avdLd1wG9SQ5uZrcADwHHmdmwmS109xHgo8AqYBPwbXffmLrkNbSyxgHwtQfU1yEik09UH8eLIl47LMnB3X1ByPqVwMokx0jDzOYD82fNmtXsQ9f0zHMj6usQkUknqsax1swuqV5pZguBweyKVL9m1DgO7Ur3GPYrb1+vvg4RmVSirpIfAz5kZj8wsy8FXz8ELgb+qjXFa70Pndabep8P3Li6+QUREWlTUU8A/K27nwZ8FtgafH3W3U8N7l/VdprROb7oXa9Lvc+efWN8a81jdZ9TRGQiSTKP437g/haUpWHufhdwV39//7gmtqxdeft6IP5JgiIiE126Bn2JdOXt61XzEJHCK1RwtHIeRxiFh4gUXaGCo1nzOLo742+vHkXhISJFVqjgaJYPv+XYho9xlcJDRAqqUMHRrKaqekZWVXPgb+9Yz1Wa5yEiBVOo4Gj1LUfijDosX/MYF9y4WuEhIoVRqOBoV8/vG9MjZ0WkMGLncUxWvT0vZuvOPU073rfWPIYB7z1peuIHRomItCPVOEJ86bwTm37M5Wse45wbHlSnuYhMaIUKjmbO45gzc2rqGx4mdeXt6jQXkYmrUMHR7M7xz8w/oSnHqaVc+1CAiMhEU6jgaLZW3HdKASIiE42CI0Zjc8iTU4CIyESh4Ihx9omvbun5lq95jHODDvQl929RiIhI2zF3z7sMTdff3+/r1q1r2vF6F61o2rHS6gAWnDJDw3hFJHNmNuju/XHbFarGkdXdcU+cnt9M9DEOzD5XLURE2oFqHAnlWeso6+owxtzp7urg6rNOYNeevQz09agmIiJNkbTGoeBIoR3Co6wyRJZfPKDwEJGGTcqmqqy9pLsz7yLsNzLmjPmB+2CpCUtEWkX3qkrhmwtP4ZwbHsy7GONU3sLk8+95g5qwRCRTCo4U5sycyonTD+eR4fweTRvnytvX7/9eISIiWVAfRx3aqa8jjdsuP43VQzsVJCJSU9I+DtU46vDuE1/NHY88kXcxUqtsZrvt8tMAFCQiklrbB4eZvRuYBxwJLHH3f8u5SFx3/pv5webtPPPcSN5FqVt1X81tl5/GnJlTGdy2S2EiIpEybaoys5uAs4Dt7v76ivVnAl8GOoEb3X1xgmNNBf7R3RfGbZt1U1XZa69ayd7R4jT13Xb5aaqViExibTGPw8zeBvwe+GY5OMysE/gF8A5gGFgLLKAUIl+oOsSH3X17sN+XgOXu/nDceVsVHAB9i1Yw1pIztd4hHTA6Vpq93gH8q2olIoXWFsERFKQXuLsiOE4F/s7dzwiWrwBw9+rQKO9vwGLgXnf/94jzXApcCjBjxow527Zta+K7iDZRO8vTmnJoJzd/+BQuuHE1e0fGxk0+VKCITGztPAHwaODxiuXhYF2YvwDeDpxrZpeFbeTuy9y93937p02b1pySJrR18byWni8vu18Y5W/vWM/z+8b2Tz5cPbQTKIXGBTeu5kv/tpkLblytyYgiBZZHcNR6xEVotcfdr3f3Oe5+mbsvjTxwRjc5TGKyhMfPntx90PK1qzYDpX6QsEDRrHaRYsljVNUwcEzF8nSgKWNb3f0u4K7+/v5LmnG8tLYunjdpmq0q1XrP167azEBfz0Gd7S8/rItHPnMGoGYtkYksjz6OLkqd438K/JpS5/j73X1jE841H5g/a9asSx599NFGD1eXwW272vK2JO1i2ku7WfqB/nE/o8oam0JFJB9t0TluZrcApwNHAL8FPuPuXzezdwHXURpJdZO7/0Mzz9vKUVW1DG7bxadvX8+m3+yO33gSmnJoJ7tfGB23fuvieeOCtzy/pEyhIpKdtgiOVmuHGkelEz+7akJPEmy1sKa+cm0kLlTK2yhYROrTzqOqMuPud7n7pYcfnt8T+yo98pkzcn16YLt68SH1/dpVN29VL5eD5dpVmznnhgdrdsirs16kcYUKjnZ0x0ffOmlGXCW1Z1+6KZNRAw5ee9XK/d8rWERao1DBkedw3DhbF8/jsK5C/bibrp4RaWlu+dKMYKncVgEjk1WhrmTt1lRVbdM1c9m6eF6xfuhNdtnb+nI795/HBEtZmsmOChgpIl3DcjC0eJ6ar0JMOeyQmut7F63g8+95Q+rjxdViXvfpe/Z/X6sBrdb+YZMdq2k2vRRVoYKjnZuqatmqABmnPBO9ll179tZc38iky+dG0t+isrqMYWVOGjCgmolMLIUKjnZvqgqjAElmoK8nl/MmCabKTvqypAFT3beS5HwKGslToYJjoisHSK2beUl4n0NZ+fkh1coX4k+ecVzk6/U0hZUl7aT/4NfXjFtX631FhUeaTvzKfRQ00iyFCo6J1lQV5leL5/H597yBww9r+wc0to3eRStiJ/zF1VjCmsKa6YFHn2r4GHGjw6rVEzTl/RQ2UkuhgmOiNlXV8v5TZvCTz5zB22YfwYsO6UAjeeN9a81jka8nCZYXhUxOrKwBxNVcmiXN8aK2TRs0UH/YlPdV4BSbLkdt7psLT+Hnfz+XLZ+fx7SXduddnJYJa66LCtB7NjwZ+lrvohWRwVKusVx0am9s2ZL2tcQ1ncVtl6W4UKonbGB84FSOWku6v0Kn/Sk4JpC1n37H/n6QotdAwnoMRsbCQ+U/YpqBooIFSjWWpQ8Mhb5evthG1VwqL8hRo6iSalatoxmSHL86YJ4bGeP1V38/0fHrGSQQdhyFT7YKfvkpri2fn1eI0Vj1dEhHdUN3dYYPLZj7+qMijxsXLGVxTWJlUTWT6oDpSDEiopFO/LAyZLnP7/eOvxNyLWkHCdTSSBNb9XEUPuEKFRxF6RxPqxwgEzFErrx9fehrYbWqiGxgbCw8Vq68fT2Xva2P3p4X13w9rsZSljRgNie8rf5AXw+dEclRffGM6sSvdaGNCpqwC/PMV9T+GdWrVQ84q7eJrdLgtl2ct7QUPuctrT98Ko9XtBAqVHAUqXO8XhM5RKp1ddb+9Rz18IvhmEcHy80PbeWxp/eEvl4OlrBbn/QuWhFbcylfJOMCprzd5t/sZl+Ke24N9PXQnaKtsp7RYh/549eEvhYVAhecMiP1ueK0+qmai+/ZRPnjGPX6wqes2XcPaJcQKlRwyMEqQ6SdgySsc3hfxKzuz90d/sDI02YdEXrhL8/kDrP0gSEee3oPNz+0NXSb958yI7LmUhYXMGVJajDVF8+xiOfoVG8bFzS1Lsy79uytaz7RCa8O/6MtLgCiQqeR8Ei779qt4y/K9Z4/zd0D4rTTLWwUHJNIuwZJWGdz1B/hUaHy/7Y8FXnhN4i8kI559PF7F62Irbn0LlqRKGCS1GCqfffhYUZS1FAgOmhqGejr4ZCUYQPxgRN1AY4KnUY1o9ZSa/JmnKR3D0iiOoTyfES1gmMSa5cgifpPHVauUQ+vqcRd+AGIuJB2GhzS1RE5THbvyFhss9jgtl37AyZsfggcXIOJujvwa64o/ZySREDlzzRJ0NT6DOLCptY+A309HFrng7oaCZ0kGt2/GZM3GynHmhq1lVY345UpOGS/6iCZPe0lB71evZy3qGGxURd+B0bHPHQi36jD1WedEHn87q6O/QET5pwbHtwfMFFBVhkwNz+0NbTc5Wv/OSdN3z9wIElXRz0Ph66nVgPsnwsTV8uqpRw6UWEcd6EM+0zTqGcwQVr1HOfhx2o3TeURHoUKjsk6qior937idGZPewkdVgqNez9x+rj/1FH/yWsJ+4s6rGZR/is7reUXD8QGS9Rw2c/dvZHBbbtCy/X8vjFOm3UEV591QmRtrTJgwgKhOmCi2sHfuvg+ADo6OrDg3zDlC0rSoKm8ACWNjOqLVnkuzNad4c14tfaDA6FzTAMjuqJm/4edt1rWNZ96j3PM1PRhnJVCBYdGVTXfvZ84naEvzOPeT5wOwC+/MG9/WHRaaTlNM9eid70u1flHPTxU3rr4vtDX4tp/44Il7gIOpb6UcsCEufqsE/YHTNIazEBfT+j7Gn7meVYP7WRkdKxUcxodS/RXdmXQJJmpXh02SWe3Jx2qXEsjoVPWSI2nLElzW9ILddzPLc0F/5qY+TutDI9CBYe0RjksfvmFAxe36gtdK/pMhp95PnabuGCJ6kOJu7VIZQ0h7DhX3r7+oIAJa+6rrMHE3VNrzdDOcUETpnfRinFBExWIlRefJLWa6n2qO/rTNPtUh85/mX1E4n3LkobPx279z9DXkoRPUmknd8bpjqnityo8FBzSNNUd7WnDpJ1GesGBPpSocsVduOHggCnX3GqprsGEnfeBR58aV5OJKmN5OG6SGg2Eh03cPgDHvWrK/tn7XZ3Gca+aErlfuekNxodO2tFmkLzGc8cjT4S+1oyaT1nc5M6kx4FSCI1EjSVPebxGKDgkU7VGbdVTO4naJuy1pP+Bwv4TJNm/ssnrsIgOhLQBE+fv7tqYqKkMSrWrWk1mUTfNrBU2Sawe2rl/9r6Peex7qaw1Vo8ue/8pMxKFVaVGajxlzaj5lCWd3Jk2hA7Jueah4JBchA0DzqLJKy5YhhoIrsq+lE3XzA3dvzJg0tRgwrat1ZkeddzP3T0+aNZ++h2h24eFTdyFPCxwkgRA9eiyuJpX5b5lWYRP0jsF1FIdQo2MGKsMoX2j3rR7ltVDwSFtp5Wh0ipJJ2s9v28sto+jLE1toDxxrLpGkzZs4vaZM3Nq6ICAuM9r9dDOxKPLwoSFTxppwwfCL/rVofP3766/g7s6hO7Z8GRu/wcUHDKhJA2VsHVRx61X5X/2pMdpRtlGffzosLjjpml2CgubKL2LVoQGTtx+A309dHWWOuQ7O5PVvMr7loWFT5paRz01nzBZhlB5OY/waPvgMLPXmdlSM/uOmV2ed3mkfUXNhG80WOrZvtq7v/KjRNs12j4dNVGz1jDkqPcVVquJ2icqcKL2O+eGBw/M6K8xaz3JxT+qb6bR8ElzjLJ6Q6jW/KVaIRRWrqzDJNPgMLObzGy7mW2oWn+mmW02sy1mtijqGO6+yd0vA84D+rMsrxRb3C1Wqju3ozq7485TyyPDzybaLs0xazWBRY3cSnt/o1q1miSimtGiOuVHxnz/zP5atZyo4ajlpzhGzZ2JOneS8IF04VFvCNXqT49rhmvl7YOyrnHcDJxZucLMOoElwFzgeGCBmR1vZm8ws7urvo4M9vkz4EfAfYhkZNM1c/eHxWFdHTU7u9uxX6XWzfeaWZsKC5u4C19Y4ER1ypfv/xUWOr/4h3eF7guli2tUU1nUucviwidOZXi0KoRaLdPgcPcHgKerVp8MbHH3IXffC9wKnO3u6939rKqv7cFx7nT304ALws5lZpea2TozW7djx46s3pIU3KZr5rJ18bzIEVLNbAZrRq0j7c33wprC6rkYRJX/y//+i7r2i6vlxIVc3MU17mIdFz5xxygfB5KFUNLwqHeIdBby6OM4Gni8Ynk4WFeTmZ1uZteb2VeBlWHbufsyd+939/5p06Y1r7QiNSRpFsh7FFja80UNS66n36WRu8nG/ZUf9d7iai1x+ye9d1iSn2+SEIqTJoRaJY/gqNVIGTpDxt1/4O5/6e4fcfclkQfWTQ6lzTQ7YLIOn3qOn7bDuNH9kvj4O4+L7ZuJ6sNKEj4Q/x6SNi8lqcE0I4SaJY/gGAaOqVieDoTP/09BNzmUiSpNx2Y7BU2j52kkPKL2vXbV5ti/yKOaIyFZ+CQpR9LmpbifYZq7KGctj+BYC8w2s2PNrBs4H7izGQdWjUMmi6yCJsnr7aTRWkuj4ZPE8/vGmhJCkLwmlLWsh+PeAjwEHGdmw2a20N1HgI8Cq4BNwLfdPfwB0imoxiFSW9qhmvX0z2RR62jUH11zb0PnT9pkFvce0oRQ3LGShlCWzFM+i7idmdl8YP6sWbMuefTRR/MujsikVM9M+ur90u5fb19K2eKVm1j6wFBDx2hGObI6VlJmNujusfPlChUcZf39/b5u3bq8iyEiKdUbOtX7Vmr1RX8ih0fS4Gj7W46IyOTRyOznRi+kzboQRx3ndZ++J9WxokZ+5fGs8bJCBYc6x0Umt0bnzjQrPMJujfLcyFiq48SN/MqLmqpERKo00mRW6xhlYbeyqedY0PzmKvVxKDhEJGeVF/x6Q6PWscoUHE2gUVUiUmTNqAlFmZTBUaYah4hIehpVJSIimShUcGhUlYhI9goVHLrliIhI9goVHCIikj0Fh4iIpKLgEBGRVLryLkAzledxAL8zs3onchwB1P/My4lJ73ly0Hsuvkbf78wkGxVyHkcjzGxdknHMRaL3PDnoPRdfq96vmqpERCQVBYeIiKSi4BhvWd4FyIHe8+Sg91x8LXm/6uMQEZFUVOMQEZFUFBwVzOxMM9tsZlvMbFHe5UnDzI4xs/vNbJOZbTSzvwrWv8LM7jWzR4N/pwbrzcyuD97rT83spIpjXRhs/6gypUFVAAAFzElEQVSZXVixfo6ZrQ/2ud7Maj/mrMXMrNPM/tPM7g6WjzWzNUH5/8XMuoP1hwbLW4LXeyuOcUWwfrOZnVGxvu1+J8zs5Wb2HTP7efB5n1r0z9nM/jr4vd5gZreY2YuK9jmb2U1mtt3MNlSsy/xzDTtHJHfXV6m5rhP4JdAHdAM/AY7Pu1wpyn8UcFLw/RTgF8DxwP8EFgXrFwFfDL5/F3APYMAAsCZY/wpgKPh3avD91OC1HwOnBvvcA8zN+30H5fo48C3g7mD528D5wfdLgcuD7/87sDT4/nzgX4Lvjw8+70OBY4Pfg852/Z0AvgFcHHzfDby8yJ8zcDTwK+Cwis/3oqJ9zsDbgJOADRXrMv9cw84RWda8/xO0y1fwA11VsXwFcEXe5Wrg/XwPeAewGTgqWHcUsDn4/qvAgortNwevLwC+WrH+q8G6o4CfV6w/aLsc3+d04D7gT4C7g/8UTwFd1Z8rsAo4Nfi+K9jOqj/r8nbt+DsBvCy4iFrV+sJ+zpSC4/HgYtgVfM5nFPFzBno5ODgy/1zDzhH1paaqA8q/nGXDwboJJ6iavxlYA7zS3Z8ECP49Mtgs7P1GrR+usT5v1wF/A4wFyz3AM+4+EixXlnP/ewtefzbYPu3PIk99wA7gfwfNczea2Uso8Ofs7r8G/hF4DHiS0uc2SLE/57JWfK5h5wil4DigVjvuhBtyZmYvBW4DPubuv4vatMY6r2N9bszsLGC7uw9Wrq6xqce8NmHeM6W/oE8CbnD3NwN/oNS8EGbCv+egzf1sSs1LrwZeAtR6eHeRPuc4ub5HBccBw8AxFcvTgSdyKktdzOwQSqGx3N2/G6z+rZkdFbx+FLA9WB/2fqPWT6+xPk9vAf7MzLYCt1JqrroOeLmZle/DVlnO/e8teP1w4GnS/yzyNAwMu/uaYPk7lIKkyJ/z24FfufsOd98HfBc4jWJ/zmWt+FzDzhFKwXHAWmB2MFKjm1Kn2p05lymxYITE14FN7v5PFS/dCZRHVlxIqe+jvP6DweiMAeDZoJq6CninmU0N/tJ7J6X23yeB3WY2EJzrgxXHyoW7X+Hu0929l9Ln9X/d/QLgfuDcYLPq91z+WZwbbO/B+vOD0TjHArMpdSS23e+Eu/8GeNzMjgtW/SnwMwr8OVNqohowsxcHZSq/58J+zhVa8bmGnSNcnp1e7fZFaaTCLyiNsLgq7/KkLPtbKVU9fwo8Eny9i1Lb7n3Ao8G/rwi2N2BJ8F7XA/0Vx/owsCX4+lDF+n5gQ7DPV6jqoM35/Z/OgVFVfZQuCFuAfwUODda/KFjeErzeV7H/VcH72kzFKKJ2/J0ATgTWBZ/1HZRGzxT6cwY+C/w8KNc/UxoZVajPGbiFUh/OPko1hIWt+FzDzhH1pZnjIiKSipqqREQkFQWHiIikouAQEZFUFBwiIpKKgkNERFJRcIjUwcxGzeyRiq+m3VHVzHor75Aq0m664jcRkRqec/cT8y6ESB5U4xBpIjPbamZfNLMfB1+zgvUzzey+4NkJ95nZjGD9K83sdjP7SfB1WnCoTjP7mpWeQfFvZnZYbm9KpIqCQ6Q+h1U1Vb2v4rXfufvJlGbnXhes+wrwTXd/I7AcuD5Yfz3wQ3d/E6V7Tm0M1s8Glrj7CcAzwDkZvx+RxDRzXKQOZvZ7d39pjfVbgT9x96HgppO/cfceM3uK0jMP9gXrn3T3I8xsBzDd3V+oOEYvcK+7zw6WPwUc4u7XZP/OROKpxiHSfB7yfdg2tbxQ8f0o6o+UNqLgEGm+91X8+1Dw/YOU7roKcAHwo+D7+4DLYf+z01/WqkKK1Et/xYjU5zAze6Ri+fvuXh6Se6iZraH0h9mCYN1fAjeZ2ScpPcHvQ8H6vwKWmdlCSjWLyyndIVWkbamPQ6SJgj6Ofnd/Ku+yiGRFTVUiIpKKahwiIpKKahwiIpKKgkNERFJRcIiISCoKDhERSUXBISIiqSg4REQklf8PK8rxhBfiM6cAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f1f8126a518>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(costs, '.')\n",
    "plt.xlabel('Epoch')\n",
    "plt.ylabel('Cost (log)')\n",
    "plt.yscale('log')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
